package cn.edu360t.tools

import cn.edu360t.config.ConfigHepler
import cn.edu360t.utils.GetJpools
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}
import redis.clients.jedis.Jedis

object Appdict2Redis {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf()
      .setMaster("local[*]")
      .setAppName("将字典数据存入到redis中")
      .set("spark.serializer", ConfigHepler.serializer)
    val sc: SparkContext = new SparkContext(conf)
    //读取数据   将appid appname存入到redis
    val result: RDD[Array[String]] = sc.textFile(ConfigHepler.appdict).map(line => line.split("\t", -1))
      .filter(_.length >= 4)
    result.foreachPartition(p => {
      // 从池子获取连接
      val jedis: Jedis = GetJpools.getJedis()
      p.foreach(arr => {
        jedis.hset("appdict", arr(4), arr(1))
      })
      jedis.close()
    })
    sc.stop()
  }
}
